import requests
import json
import math
from liebiaowang import get_pages as get_pages_from_city

host_url = "http://127.0.0.1:8008"
def get_pages():
    # 01 获取城市列表信息
    url = "{}/citys/".format(host_url)
    headers = { 'cache-control': "no-cache"}
    res = requests.request("GET", url, headers=headers)
    res = json.loads(res.text)
    total_count = res.get('count')
    page_size = 100
    # 每页100条数据, 获取真实页数
    page_count = math.ceil(total_count/page_size)

    for page_num in range(1, page_count+1):
        request_url = "{}/citys/?page_size={}&page={}".format(host_url, page_size, page_num)
        tmp_res = requests.request("GET", request_url, headers=headers)
        tmp_res = json.loads(tmp_res.text)
        tmp_results = tmp_res.get('results')
        # 02 获取城市分页信息
        for city in tmp_results:
            page_list = get_pages_from_city(city_url=city['city_url'] + 'wudaopeixun')
            for page_url in page_list:
                create_city_pages(page_data={"city": city['id'], "page_url": page_url})

def create_city_pages(page_data):
    """
    :param page_data:  {"city": 1, "page_url": "http://baidu.com"}
    :return:
    """
    url = "{}/city_pages/".format(host_url)
    page_data = json.dumps(page_data)  # 把字典转成json格式
    headers = {
        'Content-Type': "application/json",
        'cache-control': "no-cache",
    }

    response = requests.request("POST", url, data=page_data, headers=headers)
    print("城市分页数据已插入: ", response.text)


if __name__ == '__main__':
    get_pages()


